
#include "CDBN_Image.h"

#define FLAG_USE_GPU_IMPLEMENTATION				1
#define FLAG_USE_FFT_FOR_ConDiv					0
#define CONVOLUTION_USE_FFT_THRESHOLD			200.0
#define	FLAG_MAXPOOLING_DESERT_UNITS			1

#if FLAG_USE_GPU_IMPLEMENTATION
#include "../../HiBoost/CDBN_GPU/GPU_Mat.h"
#include "../../HiBoost/CDBN_GPU/GPU_Convolution.h"
using namespace gpu_tianqichen;
#else
#define __NOT_USE_GPU__
#endif

// [*] nw_rbm::CConvolutionalDBN_for_Image::CBasicLayerInfo

void nw_rbm::CConvolutionalDBN_for_Image::CBasicLayerInfo::myInit( int K_Visible, int K_Hidden, int wMask, int hMask, int xPool, int yPool )
{
	aMask = wMask * hMask;
	sMask = aMask * K_Visible * K_Hidden;
}

void nw_rbm::CConvolutionalDBN_for_Image::CBasicLayerInfo::LoadFromStream( istream& inStream )
{
	inStream>>K_Visible;
	LoadFromStream(inStream, K_Visible);
}

void nw_rbm::CConvolutionalDBN_for_Image::CBasicLayerInfo::LoadFromStream( istream& inStream, int K_Visible )
{
	this->K_Visible = K_Visible;
	inStream>>K_Hidden;
	inStream>>wMask>>hMask;
	inStream>>xPool>>yPool;
	myInit(K_Visible, K_Hidden, wMask, hMask, xPool, yPool);
}

void nw_rbm::CConvolutionalDBN_for_Image::CBasicLayerInfo::SaveToStream( ostream& outStream ) const
{
	outStream<<K_Visible<<" "<<K_Hidden;
	outStream<<"\t"<<wMask<<" "<<hMask;
	outStream<<"\t"<<xPool<<" "<<yPool;
	outStream<<endl;
}

// [*] nw_rbm::CConvolutionalDBN_for_Image::CExtraLayerInfo

nw_rbm::CConvolutionalDBN_for_Image::CExtraLayerInfo::CExtraLayerInfo()
{
	Clear();
}

void nw_rbm::CConvolutionalDBN_for_Image::CExtraLayerInfo::Init_RBM_InferenceHelper(
	int LayerIndex,
	C_RBM_InferenceHelper*& p_RBM_InferenceHelper)
{
	if (0)
	{
		p_RBM_InferenceHelper = new C_RBM_InferenceHelper;
	}
	else
	{
		C_RBM_InferenceHelper_Convolution* p_RBM_InferenceHelper_Convolution = new C_RBM_InferenceHelper_Convolution;
		p_RBM_InferenceHelper_Convolution->qExtraLayerInfo = this;
		p_RBM_InferenceHelper = p_RBM_InferenceHelper_Convolution;
	}
}

void nw_rbm::CConvolutionalDBN_for_Image::CExtraLayerInfo::Clear()
{
	this->myInit(0, 0, 0, 0, 0, 0);
	wVisible = hVisible = aVisible = sVisible = 0;
	wHidden = hHidden = aHidden = sHidden = 0;
	wPool = hPool = aPool = sPool = 0;
}

int nw_rbm::CConvolutionalDBN_for_Image::CExtraLayerInfo::GetAmountConnection_Visible(
	int UnitHandle ) const
{
	if (UnitHandle<0 || UnitHandle>=sVisible) return 0;

	int kVisible, xVisible, yVisible;
	int xHead, yHead, xTail, yTail;
	kVisible = UnitHandle / aVisible;
	UnitHandle %= aVisible;
	xVisible = UnitHandle % wVisible;
	yVisible = UnitHandle / wVisible;
	xHead = xVisible - (wMask - 1); if (xHead < 0) xHead = 0;
	yHead = yVisible - (hMask - 1); if (yHead < 0) yHead = 0;
	xTail = xVisible + 1; if (xTail > wHidden) xTail = wHidden;
	yTail = yVisible + 1; if (yTail > hHidden) yTail = hHidden;
	int ret = (xTail - xHead) * (yTail - yHead) * K_Hidden;

	return ret;
}

int nw_rbm::CConvolutionalDBN_for_Image::CExtraLayerInfo::GetUnitConnectionList_Visible(
	int UnitHandle,
	CSimpleTypeArray<int>& vConnectedUnitHandle,
	CSimpleTypeArray<int>& vEdgeHandle ) const
{
	if (UnitHandle<0 || UnitHandle>=sVisible) return 0;

	int kVisible, xVisible, yVisible;
	int xHead, yHead, xTail, yTail;
	kVisible = UnitHandle / aVisible;
	UnitHandle %= aVisible;
	xVisible = UnitHandle % wVisible;
	yVisible = UnitHandle / wVisible;
	xHead = xVisible - (wMask - 1); if (xHead < 0) xHead = 0;
	yHead = yVisible - (hMask - 1); if (yHead < 0) yHead = 0;
	xTail = xVisible + 1; if (xTail > wHidden) xTail = wHidden;
	yTail = yVisible + 1; if (yTail > hHidden) yTail = hHidden;
	int ret = (xTail - xHead) * (yTail - yHead) * K_Hidden;

	vConnectedUnitHandle.Resize(ret); vConnectedUnitHandle.Clear();
	vEdgeHandle.Resize(ret); vEdgeHandle.Clear();
	int kHidden, xHidden, yHidden;
	int UI_Hidden, WI_Result, UI_Offset, WI_Offset;
	for ( kHidden = 0; kHidden < K_Hidden; ++kHidden )
	{
		UI_Offset = kHidden * aHidden;
		WI_Offset = aMask * (kVisible + K_Visible * kHidden);
		for ( yHidden = yHead; yHidden < yTail; ++yHidden ) for ( xHidden = xHead; xHidden < xTail; ++xHidden )
		{
			UI_Hidden = xHidden + yHidden * wHidden + UI_Offset;
			WI_Result = (xVisible - xHidden) + (yVisible - yHidden) * wMask + WI_Offset;
			vConnectedUnitHandle.PushBack(UI_Hidden);
			vEdgeHandle.PushBack(WI_Result);
		}
	}

	return ret;
}

int nw_rbm::CConvolutionalDBN_for_Image::CExtraLayerInfo::GetAmountConnection_Hidden(
	int UnitHandle ) const
{
	if (UnitHandle<0 || UnitHandle>=sHidden) return 0;

	//int kHidden, xHidden, yHidden;
	//kHidden = UnitHandle / aHidden;
	//UnitHandle %= aHidden;
	//xHidden = UnitHandle % wHidden;
	//yHidden = UnitHandle / wHidden;
	int ret = aMask * K_Visible;

	return ret;
}

int nw_rbm::CConvolutionalDBN_for_Image::CExtraLayerInfo::GetUnitConnectionList_Hidden(
	int UnitHandle,
	CSimpleTypeArray<int>& vConnectedUnitHandle,
	CSimpleTypeArray<int>& vEdgeHandle ) const
{
	if (UnitHandle<0 || UnitHandle>=sHidden) return 0;

	int kHidden, xHidden, yHidden;
	kHidden = UnitHandle / aHidden;
	UnitHandle %= aHidden;
	xHidden = UnitHandle % wHidden;
	yHidden = UnitHandle / wHidden;
	int ret = aMask * K_Visible;

	vConnectedUnitHandle.Resize(ret); vConnectedUnitHandle.Clear();
	vEdgeHandle.Resize(ret); vEdgeHandle.Clear();
	int kVisible, ox, oy;
	int UI_Visible, WI_Result, UI_Offset, WI_Offset;
	for ( kVisible = 0; kVisible < K_Visible; ++kVisible )
	{
		UI_Offset = kVisible * aVisible;
		WI_Offset = aMask * (kVisible + K_Visible * kHidden);
		for ( oy = 0; oy < hMask; ++oy ) for ( ox = 0; ox < wMask; ++ox )
		{
			UI_Visible = (xHidden+ox) + (yHidden+oy) * wVisible + UI_Offset;
			WI_Result = ox + oy * wMask + WI_Offset;
			vConnectedUnitHandle.PushBack(UI_Visible);
			vEdgeHandle.PushBack(WI_Result);
		}
	}

	return ret;
}

// [*] nw_rbm::CConvolutionalDBN_for_Image::C_RBM_InferenceHelper_Convolution

nw_rbm::CConvolutionalDBN_for_Image::C_RBM_InferenceHelper_Convolution::~C_RBM_InferenceHelper_Convolution()
{
}

void nw_rbm::CConvolutionalDBN_for_Image::C_RBM_InferenceHelper_Convolution::InferenceEnergy_of_Units_Visible(CRestrictedBoltzmannMachine* pRBM)
{
	if (pRBM == NULL) return;

	bool fUseFFT = (qExtraLayerInfo != NULL);
	if (fUseFFT)
	{
		RBM_FLOAT zp = (RBM_FLOAT)loirey::fft::CostConvolution_2D(
			0, 0, qExtraLayerInfo->wVisible, qExtraLayerInfo->hVisible,
			false, false, true, true,
			qExtraLayerInfo->wMask, qExtraLayerInfo->hMask,
			qExtraLayerInfo->wMask - 1, qExtraLayerInfo->hMask - 1);
		fUseFFT = (zp < CONVOLUTION_USE_FFT_THRESHOLD);
	}

	if (!fUseFFT)
	{
		C_RBM_InferenceHelper tHelper;
		tHelper.InferenceEnergy_of_Units_Visible(pRBM);
	}
	else
	{
#if FLAG_USE_GPU_IMPLEMENTATION
		CSimpleTypeArray<GPU_FLOAT> mat4D_Mask;
		CSimpleTypeArray<GPU_FLOAT> mat3D_V;
		CSimpleTypeArray<GPU_FLOAT> mat3D_H;
		CSimpleTypeArray<GPU_FLOAT> Bias_V;
		CSimpleTypeArray<GPU_FLOAT> Bias_H;
		mat4D_Mask.Resize(qExtraLayerInfo->sMask);
		mat3D_H.Resize(qExtraLayerInfo->sHidden);
		mat3D_V.Resize(qExtraLayerInfo->sVisible);
		Bias_V.Resize(qExtraLayerInfo->K_Visible);
		Bias_H.Resize(qExtraLayerInfo->K_Hidden);
		int i_k_h, i_k_v, i_pixel;
		GPU_FLOAT* pGPU_FLOAT;

		pGPU_FLOAT = mat4D_Mask.pElement;
		for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v ) for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		{
			const CWeightItem* qWI = pRBM->ListWeightItem_Edge.pElement
				+ qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
			for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
				*(pGPU_FLOAT++) = (GPU_FLOAT)((qWI++)->Value);
		}
		pGPU_FLOAT = mat3D_H.pElement;
		for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		{
			const CUnitBasic* qUH = pRBM->ListUnit_Hidden.pElement
				+ qExtraLayerInfo->aHidden * i_k_h;
			for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
				*(pGPU_FLOAT++) = (GPU_FLOAT)((qUH++)->State);
		}
		pGPU_FLOAT = Bias_V.pElement;
		for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
			*(pGPU_FLOAT++) = (GPU_FLOAT)pRBM->ListWeightItem_Visible.pElement[i_k_v].Value;
		//pGPU_FLOAT = mat3D_V.pElement;
		//for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
		//{
		//	const CUnitBasic* qUV = pRBM->ListUnit_Visible.pElement
		//		+ qExtraLayerInfo->aVisible * i_k_v;
		//	for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
		//		*(pGPU_FLOAT++) = (GPU_FLOAT)((qUV++)->State);
		//}
		//pGPU_FLOAT = Bias_H.pElement;
		//for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		//	*(pGPU_FLOAT++) = (GPU_FLOAT)pRBM->ListWeightItem_Hidden.pElement[i_k_h].Value;

		Mat3D gpu_mat3D_V(
			qExtraLayerInfo->K_Visible,
			qExtraLayerInfo->hVisible,
			qExtraLayerInfo->wVisible);
		gpu_mat3D_V.elem = mat3D_V.pElement;
		gpu_mat3D_V.pitch = sizeof(GPU_FLOAT) * gpu_mat3D_V.x_max;
		Mat3D gpu_mat3D_H(
			qExtraLayerInfo->K_Hidden,
			qExtraLayerInfo->hHidden, 
			qExtraLayerInfo->wHidden);
		gpu_mat3D_H.elem = mat3D_H.pElement;
		gpu_mat3D_H.pitch = sizeof(GPU_FLOAT) * gpu_mat3D_H.x_max;
		Mat4DBias gpu_mat4Dbias_M(
			qExtraLayerInfo->K_Visible,
			qExtraLayerInfo->K_Hidden,
			qExtraLayerInfo->hMask,
			qExtraLayerInfo->wMask);
		gpu_mat4Dbias_M.mat.elem = mat4D_Mask.pElement;
		gpu_mat4Dbias_M.mat.pitch = sizeof(GPU_FLOAT) * gpu_mat4Dbias_M.mat.x_max;
		gpu_mat4Dbias_M.v_bias = Bias_V.pElement;
		gpu_mat4Dbias_M.h_bias = Bias_H.pElement;

		// Convolution_H2V
		if (1)
		{
			//TODO
			convolution_h2v(gpu_mat3D_V, gpu_mat3D_H, gpu_mat4Dbias_M);
		} // Convolution_H2V

		//pGPU_FLOAT = mat3D_H.pElement;
		//for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		//{
		//	CUnitBasic* pUH = pRBM->ListUnit_Hidden.pElement
		//		+ qExtraLayerInfo->aHidden * i_k_h;
		//	for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
		//		(pUH++)->Energy = (RBM_FLOAT)*(pGPU_FLOAT++);
		//}
		pGPU_FLOAT = mat3D_V.pElement;
		for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
		{
			CUnitBasic* pUV = pRBM->ListUnit_Visible.pElement
				+ qExtraLayerInfo->aVisible * i_k_v;
			for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
				(pUV++)->Energy = (RBM_FLOAT)*(pGPU_FLOAT++);
		}
#else
		int i_k_h, i_k_v, i_pixel;
		CSimpleTypeArray<RBM_FLOAT> matMask;
		CSimpleTypeArray<RBM_FLOAT> matData;
		CSimpleTypeArray<RBM_FLOAT> matResult;
		const CWeightItem* qWI;
		const CUnitBasic* qUH;
		CUnitBasic* pUV;
		matMask.Resize(qExtraLayerInfo->aMask);
		matData.Resize(qExtraLayerInfo->aHidden);
		matResult.Resize(qExtraLayerInfo->aVisible);

		for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
		{
			pUV = pRBM->ListUnit_Visible.pElement + qExtraLayerInfo->aVisible * i_k_v;
			qWI = pRBM->ListWeightItem_Visible.pElement + i_k_v;
			if (pUV->BiasIndex >= 0)
			{
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
					pUV[i_pixel].Energy = qWI->Value;
			}
			else
			{
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
					pUV[i_pixel].Energy = 0;
			}
			for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
			{
				qWI = pRBM->ListWeightItem_Edge.pElement + qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
				qUH = pRBM->ListUnit_Hidden.pElement + qExtraLayerInfo->aHidden * i_k_h;
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
					matMask.pElement[i_pixel] = qWI[qExtraLayerInfo->aMask - 1 - i_pixel].Value;
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
					matData.pElement[i_pixel] = qUH[i_pixel].State;
				loirey::fft::Convolution_2D(
					matData.pElement, qExtraLayerInfo->wHidden, qExtraLayerInfo->hHidden,
					loirey::fft::BORDER_MODE_Zero,
					0, 0, qExtraLayerInfo->wVisible, qExtraLayerInfo->hVisible,
					false, false, true, true,
					matMask.pElement, qExtraLayerInfo->wMask, qExtraLayerInfo->hMask,
					qExtraLayerInfo->wMask - 1, qExtraLayerInfo->hMask - 1,
					matResult.pElement);
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
					pUV[i_pixel].Energy += matResult.pElement[i_pixel];
			}
		}
#endif
	}
}

void nw_rbm::CConvolutionalDBN_for_Image::C_RBM_InferenceHelper_Convolution::InferenceEnergy_of_Units_Hidden(CRestrictedBoltzmannMachine* pRBM)
{
	if (pRBM == NULL) return;

	bool fUseFFT = (qExtraLayerInfo != NULL);
	if (fUseFFT)
	{
		RBM_FLOAT zp = (RBM_FLOAT)loirey::fft::CostConvolution_2D(
			0, 0, qExtraLayerInfo->wHidden, qExtraLayerInfo->hHidden,
			true, true, true, true,
			qExtraLayerInfo->wMask, qExtraLayerInfo->hMask,
			0, 0);
		fUseFFT = (zp < CONVOLUTION_USE_FFT_THRESHOLD);
	}

	if (!fUseFFT)
	{
		C_RBM_InferenceHelper tHelper;
		tHelper.InferenceEnergy_of_Units_Hidden(pRBM);
	}
	else
	{
#if FLAG_USE_GPU_IMPLEMENTATION
		CSimpleTypeArray<GPU_FLOAT> mat4D_Mask;
		CSimpleTypeArray<GPU_FLOAT> mat3D_V;
		CSimpleTypeArray<GPU_FLOAT> mat3D_H;
		CSimpleTypeArray<GPU_FLOAT> Bias_V;
		CSimpleTypeArray<GPU_FLOAT> Bias_H;
		mat4D_Mask.Resize(qExtraLayerInfo->sMask);
		mat3D_H.Resize(qExtraLayerInfo->sHidden);
		mat3D_V.Resize(qExtraLayerInfo->sVisible);
		Bias_V.Resize(qExtraLayerInfo->K_Visible);
		Bias_H.Resize(qExtraLayerInfo->K_Hidden);
		int i_k_h, i_k_v, i_pixel;
		GPU_FLOAT* pGPU_FLOAT;

		pGPU_FLOAT = mat4D_Mask.pElement;
		for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v ) for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		{
			const CWeightItem* qWI = pRBM->ListWeightItem_Edge.pElement
				+ qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
			for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
				*(pGPU_FLOAT++) = (GPU_FLOAT)((qWI++)->Value);
		}
		//pGPU_FLOAT = mat3D_H.pElement;
		//for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		//{
		//	const CUnitBasic* qUH = pRBM->ListUnit_Hidden.pElement
		//		+ qExtraLayerInfo->aHidden * i_k_h;
		//	for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
		//		*(pGPU_FLOAT++) = (GPU_FLOAT)((qUH++)->State);
		//}
		//pGPU_FLOAT = Bias_V.pElement;
		//for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
		//	*(pGPU_FLOAT++) = (GPU_FLOAT)pRBM->ListWeightItem_Visible.pElement[i_k_v].Value;
		pGPU_FLOAT = mat3D_V.pElement;
		for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
		{
			const CUnitBasic* qUV = pRBM->ListUnit_Visible.pElement
				+ qExtraLayerInfo->aVisible * i_k_v;
			for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
				*(pGPU_FLOAT++) = (GPU_FLOAT)((qUV++)->State);
		}
		pGPU_FLOAT = Bias_H.pElement;
		for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
			*(pGPU_FLOAT++) = (GPU_FLOAT)pRBM->ListWeightItem_Hidden.pElement[i_k_h].Value;

		Mat3D gpu_mat3D_V(
			qExtraLayerInfo->K_Visible,
			qExtraLayerInfo->hVisible,
			qExtraLayerInfo->wVisible);
		gpu_mat3D_V.elem = mat3D_V.pElement;
		gpu_mat3D_V.pitch = sizeof(GPU_FLOAT) * gpu_mat3D_V.x_max;
		Mat3D gpu_mat3D_H(
			qExtraLayerInfo->K_Hidden,
			qExtraLayerInfo->hHidden, 
			qExtraLayerInfo->wHidden);
		gpu_mat3D_H.elem = mat3D_H.pElement;
		gpu_mat3D_H.pitch = sizeof(GPU_FLOAT) * gpu_mat3D_H.x_max;
		Mat4DBias gpu_mat4Dbias_M(
			qExtraLayerInfo->K_Visible,
			qExtraLayerInfo->K_Hidden,
			qExtraLayerInfo->hMask,
			qExtraLayerInfo->wMask);
		gpu_mat4Dbias_M.mat.elem = mat4D_Mask.pElement;
		gpu_mat4Dbias_M.mat.pitch = sizeof(GPU_FLOAT) * gpu_mat4Dbias_M.mat.x_max;
		gpu_mat4Dbias_M.v_bias = Bias_V.pElement;
		gpu_mat4Dbias_M.h_bias = Bias_H.pElement;

		// Convolution_V2H
		if (1)
		{
			//TODO
			convolution_v2h(gpu_mat3D_H, gpu_mat3D_V, gpu_mat4Dbias_M);
		} // Convolution_V2H

		pGPU_FLOAT = mat3D_H.pElement;
		for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		{
			CUnitBasic* pUH = pRBM->ListUnit_Hidden.pElement
				+ qExtraLayerInfo->aHidden * i_k_h;
			for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
				(pUH++)->Energy = (RBM_FLOAT)*(pGPU_FLOAT++);
		}
		//pGPU_FLOAT = mat3D_V.pElement;
		//for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
		//{
		//	CUnitBasic* pUV = pRBM->ListUnit_Visible.pElement
		//		+ qExtraLayerInfo->aVisible * i_k_v;
		//	for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
		//		(pUV++)->Energy = (RBM_FLOAT)*(pGPU_FLOAT++);
		//}
#else
		int i_k_v, i_k_h, i_pixel;
		CSimpleTypeArray<RBM_FLOAT> matMask;
		CSimpleTypeArray<RBM_FLOAT> matData;
		CSimpleTypeArray<RBM_FLOAT> matResult;
		const CWeightItem* qWI;
		const CUnitBasic* qUV;
		CUnitBasic* pUH;
		matMask.Resize(qExtraLayerInfo->aMask);
		matData.Resize(qExtraLayerInfo->aVisible);
		matResult.Resize(qExtraLayerInfo->aHidden);

		for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		{
			pUH = pRBM->ListUnit_Hidden.pElement + qExtraLayerInfo->aHidden * i_k_h;
			qWI = pRBM->ListWeightItem_Hidden.pElement + i_k_h;
			if (pUH->BiasIndex >= 0)
			{
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
					pUH[i_pixel].Energy = qWI->Value;
			}
			else
			{
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
					pUH[i_pixel].Energy = 0.0f;
			}
			for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
			{
				qWI = pRBM->ListWeightItem_Edge.pElement + qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
				qUV = pRBM->ListUnit_Visible.pElement + qExtraLayerInfo->aVisible * i_k_v;
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
					matMask.pElement[i_pixel] = qWI[i_pixel].Value;
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
					matData.pElement[i_pixel] = qUV[i_pixel].State;
				loirey::fft::Convolution_2D(
					matData.pElement, qExtraLayerInfo->wVisible, qExtraLayerInfo->hVisible,
					loirey::fft::BORDER_MODE_Zero,
					0, 0, qExtraLayerInfo->wHidden, qExtraLayerInfo->hHidden,
					true, true, true, true,
					matMask.pElement, qExtraLayerInfo->wMask, qExtraLayerInfo->hMask,
					0, 0,
					matResult.pElement);
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
					pUH[i_pixel].Energy += matResult.pElement[i_pixel];
			}
		}
#endif
	}
}

RBM_FLOAT nw_rbm::CConvolutionalDBN_for_Image::C_RBM_InferenceHelper_Convolution::ComputeEdgeGradient_ConDiv(
	CRestrictedBoltzmannMachine* pRBM,
	const RBM_FLOAT* qVisibleState_Data,
	const RBM_FLOAT* qHiddenExpectation_Data,
	const RBM_FLOAT* qVisibleState_Model,
	const RBM_FLOAT* qHiddenExpectation_Model,
	RBM_FLOAT ExampleWeight, RBM_FLOAT GradientAmendment )
{
	RBM_FLOAT VisibleError = 0.0f;

	if (pRBM==NULL
		|| qVisibleState_Data==NULL
		|| qHiddenExpectation_Data==NULL
		|| qVisibleState_Model==NULL
		|| qHiddenExpectation_Model==NULL
		|| abs(ExampleWeight)<EPS
		)
		return VisibleError;

	bool fUseFFT = (qExtraLayerInfo != NULL);
	if (fUseFFT)
	{
		fUseFFT = !!FLAG_USE_FFT_FOR_ConDiv || !!FLAG_USE_GPU_IMPLEMENTATION;

		// Tianqi Chen
		//fUseFFT = false;

		//RBM_FLOAT zp = (RBM_FLOAT)loirey::fft::CostConvolution_2D(
		//	0, 0, qExtraLayerInfo->wHidden, qExtraLayerInfo->hHidden,
		//	true, true, true, true,
		//	qExtraLayerInfo->wMask, qExtraLayerInfo->hMask,
		//	0, 0);
		//fUseFFT = (zp < CONVOLUTION_USE_FFT_THRESHOLD);
	}

	if (!fUseFFT)
	{
		C_RBM_InferenceHelper tHelper;
		return tHelper.ComputeEdgeGradient_ConDiv(
			pRBM,
			qVisibleState_Data,
			qHiddenExpectation_Data,
			qVisibleState_Model,
			qHiddenExpectation_Model,
			ExampleWeight, GradientAmendment);
	}
	else
	{
		int i_V, j_H, o;
		int i_k_h, i_k_v, i_pixel;
		RBM_FLOAT dw;
		CWeightItem* pWI;

		GradientAmendment *= ExampleWeight;

		pWI = pRBM->ListWeightItem_Visible.pElement;
		for ( i_V = 0; i_V < pRBM->AmountUnit_Visible; ++i_V )
		{
			const CUnitBasic& tVi = pRBM->ListUnit_Visible.pElement[i_V];
			o = tVi.BiasIndex;
			dw = qVisibleState_Data[i_V] - qVisibleState_Model[i_V];
			if (o >= 0)
			{
				pWI[o].ConDiv_NextIncrease += dw * GradientAmendment;
				pWI[o].ConDiv_CumulativeWeight += ExampleWeight;
			}
			VisibleError += dw * dw;
		}
		VisibleError = sqrt(VisibleError / pRBM->AmountUnit_Visible);
		VisibleError *= ExampleWeight;

		pWI = pRBM->ListWeightItem_Hidden.pElement;
		for ( j_H = 0; j_H < pRBM->AmountUnit_Hidden; ++j_H )
		{
			const CUnitBasic& tHj = pRBM->ListUnit_Hidden.pElement[j_H];
			o = tHj.BiasIndex;
			dw = qHiddenExpectation_Data[j_H] - qHiddenExpectation_Model[j_H];
			if (o >= 0)
			{
				pWI[o].ConDiv_NextIncrease += dw * GradientAmendment;
				pWI[o].ConDiv_CumulativeWeight += ExampleWeight;
			}
		}

#if FLAG_USE_GPU_IMPLEMENTATION
		//CSimpleTypeArray<RBM_FLOAT> matDeltaMask;
		//const RBM_FLOAT* qVs;
		//const RBM_FLOAT* qPHs;
		//const RBM_FLOAT* qVt;
		//const RBM_FLOAT* qPHt;
		//matDeltaMask.Resize(qExtraLayerInfo->aMask);
		//CSimpleTypeArray<double> vTemp_Data;
		//vTemp_Data.Resize(qExtraLayerInfo->sMask);
		//CSimpleTypeArray<double> vTemp_Model;
		//vTemp_Model.Resize(qExtraLayerInfo->sMask);
		//double* ppp;
		//for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		//{
		//	qPHs = qHiddenExpectation_Data + qExtraLayerInfo->aHidden * i_k_h;
		//	qPHt = qHiddenExpectation_Model + qExtraLayerInfo->aHidden * i_k_h;
		//	for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
		//	{
		//		ppp = vTemp_Data.pElement + qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
		//		qVs = qVisibleState_Data + qExtraLayerInfo->aVisible * i_k_v;
		//		qVt = qVisibleState_Model + qExtraLayerInfo->aVisible * i_k_v;

		//		loirey::fft::Convolution_2D(
		//			qVs, qExtraLayerInfo->wVisible, qExtraLayerInfo->hVisible,
		//			loirey::fft::BORDER_MODE_Zero,
		//			0, 0, qExtraLayerInfo->wMask, qExtraLayerInfo->hMask,
		//			false, false, true, true,
		//			qPHs, qExtraLayerInfo->wHidden, qExtraLayerInfo->hHidden,
		//			0, 0,
		//			matDeltaMask.pElement);
		//		for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
		//			ppp[i_pixel] = matDeltaMask.pElement[i_pixel] * GradientAmendment;

		//		ppp = vTemp_Model.pElement + qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
		//		loirey::fft::Convolution_2D(
		//			qVt, qExtraLayerInfo->wVisible, qExtraLayerInfo->hVisible,
		//			loirey::fft::BORDER_MODE_Zero,
		//			0, 0, qExtraLayerInfo->wMask, qExtraLayerInfo->hMask,
		//			false, false, true, true,
		//			qPHt, qExtraLayerInfo->wHidden, qExtraLayerInfo->hHidden,
		//			0, 0,
		//			matDeltaMask.pElement);
		//		for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
		//			ppp[i_pixel] = 0.0 - matDeltaMask.pElement[i_pixel] * GradientAmendment;
		//	} // i_k_v
		//} // i_k_h

		CSimpleTypeArray<GPU_FLOAT> mat4D_Mask;
		CSimpleTypeArray<GPU_FLOAT> mat3D_V;
		CSimpleTypeArray<GPU_FLOAT> mat3D_H;
		CSimpleTypeArray<GPU_FLOAT> Bias_V;
		CSimpleTypeArray<GPU_FLOAT> Bias_H;
		mat4D_Mask.Resize(qExtraLayerInfo->sMask);
		mat3D_H.Resize(qExtraLayerInfo->sHidden);
		mat3D_V.Resize(qExtraLayerInfo->sVisible);
		Bias_V.Resize(qExtraLayerInfo->K_Visible);
		Bias_H.Resize(qExtraLayerInfo->K_Hidden);
		GPU_FLOAT* pGPU_FLOAT;

		Mat3D gpu_mat3D_V(
			qExtraLayerInfo->K_Visible,
			qExtraLayerInfo->hVisible,
			qExtraLayerInfo->wVisible);
		gpu_mat3D_V.elem = mat3D_V.pElement;
		gpu_mat3D_V.pitch = sizeof(GPU_FLOAT) * gpu_mat3D_V.x_max;
		Mat3D gpu_mat3D_H(
			qExtraLayerInfo->K_Hidden,
			qExtraLayerInfo->hHidden, 
			qExtraLayerInfo->wHidden);
		gpu_mat3D_H.elem = mat3D_H.pElement;
		gpu_mat3D_H.pitch = sizeof(GPU_FLOAT) * gpu_mat3D_H.x_max;
		Mat4DBias gpu_mat4Dbias_M(
			qExtraLayerInfo->K_Visible,
			qExtraLayerInfo->K_Hidden,
			qExtraLayerInfo->hMask,
			qExtraLayerInfo->wMask);
		gpu_mat4Dbias_M.mat.elem = mat4D_Mask.pElement;
		gpu_mat4Dbias_M.mat.pitch = sizeof(GPU_FLOAT) * gpu_mat4Dbias_M.mat.x_max;
		gpu_mat4Dbias_M.v_bias = Bias_V.pElement;
		gpu_mat4Dbias_M.h_bias = Bias_H.pElement;

		//pGPU_FLOAT = Bias_V.pElement;
		//for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
		//	*(pGPU_FLOAT++) = (GPU_FLOAT)pRBM->ListWeightItem_Visible.pElement[i_k_v].Value;
		//pGPU_FLOAT = Bias_H.pElement;
		//for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		//	*(pGPU_FLOAT++) = (GPU_FLOAT)pRBM->ListWeightItem_Hidden.pElement[i_k_h].Value;

		// Convolution_A2M
		if (1)
		{
			//TODO

			pGPU_FLOAT = mat3D_V.pElement;
			for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
			{
				const RBM_FLOAT* qVS = qVisibleState_Data
					+ qExtraLayerInfo->aVisible * i_k_v;
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
					*(pGPU_FLOAT++) = (GPU_FLOAT)*(qVS++);
			}
			pGPU_FLOAT = mat3D_H.pElement;
			for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
			{
				const RBM_FLOAT* qHE = qHiddenExpectation_Data
					+ qExtraLayerInfo->aHidden * i_k_h;
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
					*(pGPU_FLOAT++) = (GPU_FLOAT)*(qHE++);
			}
			convolution_a2m(gpu_mat4Dbias_M, gpu_mat3D_V, gpu_mat3D_H);
			pGPU_FLOAT = mat4D_Mask.pElement;
			for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v ) for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
			{
				CWeightItem* pWI = pRBM->ListWeightItem_Edge.pElement
					+ qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
				//ppp = vTemp_Data.pElement + qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
				{
					pWI->ConDiv_NextIncrease += GradientAmendment * (RBM_FLOAT)*(pGPU_FLOAT++);
					pWI->ConDiv_CumulativeWeight += ExampleWeight * qExtraLayerInfo->aHidden;
					++pWI;
				}
			}

			pGPU_FLOAT = mat3D_V.pElement;
			for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
			{
				const RBM_FLOAT* qVS = qVisibleState_Model
					+ qExtraLayerInfo->aVisible * i_k_v;
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aVisible; ++i_pixel )
					*(pGPU_FLOAT++) = (GPU_FLOAT)*(qVS++);
			}
			pGPU_FLOAT = mat3D_H.pElement;
			for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
			{
				const RBM_FLOAT* qHE = qHiddenExpectation_Model
					+ qExtraLayerInfo->aHidden * i_k_h;
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aHidden; ++i_pixel )
					*(pGPU_FLOAT++) = (GPU_FLOAT)*(qHE++);
			}
			convolution_a2m(gpu_mat4Dbias_M, gpu_mat3D_V, gpu_mat3D_H);
			pGPU_FLOAT = mat4D_Mask.pElement;
			for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v ) for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
			{
				CWeightItem* pWI = pRBM->ListWeightItem_Edge.pElement
					+ qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
				//ppp = vTemp_Model.pElement + qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
				{
					(pWI++)->ConDiv_NextIncrease -= GradientAmendment * (RBM_FLOAT)*(pGPU_FLOAT++);
				}
			}
		} // Convolution_V2H
#else
		CSimpleTypeArray<RBM_FLOAT> matDeltaMask;
		const RBM_FLOAT* qVs;
		const RBM_FLOAT* qPHs;
		const RBM_FLOAT* qVt;
		const RBM_FLOAT* qPHt;
		matDeltaMask.Resize(qExtraLayerInfo->aMask);
		for ( i_k_h = 0; i_k_h < qExtraLayerInfo->K_Hidden; ++i_k_h )
		{
			qPHs = qHiddenExpectation_Data + qExtraLayerInfo->aHidden * i_k_h;
			qPHt = qHiddenExpectation_Model + qExtraLayerInfo->aHidden * i_k_h;
			for ( i_k_v = 0; i_k_v < qExtraLayerInfo->K_Visible; ++i_k_v )
			{
				pWI = pRBM->ListWeightItem_Edge.pElement + qExtraLayerInfo->aMask * (i_k_v + qExtraLayerInfo->K_Visible * i_k_h);
				qVs = qVisibleState_Data + qExtraLayerInfo->aVisible * i_k_v;
				qVt = qVisibleState_Model + qExtraLayerInfo->aVisible * i_k_v;

				loirey::fft::Convolution_2D(
					qVs, qExtraLayerInfo->wVisible, qExtraLayerInfo->hVisible,
					loirey::fft::BORDER_MODE_Zero,
					0, 0, qExtraLayerInfo->wMask, qExtraLayerInfo->hMask,
					false, false, true, true,
					qPHs, qExtraLayerInfo->wHidden, qExtraLayerInfo->hHidden,
					0, 0,
					matDeltaMask.pElement);
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
				{
					pWI[i_pixel].ConDiv_NextIncrease += matDeltaMask.pElement[i_pixel] * GradientAmendment;
					pWI[i_pixel].ConDiv_CumulativeWeight += ExampleWeight * qExtraLayerInfo->aHidden;
				} // i_pixel

				loirey::fft::Convolution_2D(
					qVt, qExtraLayerInfo->wVisible, qExtraLayerInfo->hVisible,
					loirey::fft::BORDER_MODE_Zero,
					0, 0, qExtraLayerInfo->wMask, qExtraLayerInfo->hMask,
					false, false, true, true,
					qPHt, qExtraLayerInfo->wHidden, qExtraLayerInfo->hHidden,
					0, 0,
					matDeltaMask.pElement);
				for ( i_pixel = 0; i_pixel < qExtraLayerInfo->aMask; ++i_pixel )
				{
					pWI[i_pixel].ConDiv_NextIncrease -= matDeltaMask.pElement[i_pixel] * GradientAmendment;
				} // i_pixel

			} // i_k_v
		} // i_k_h
#endif
	} // (!fUseFFT)

	return VisibleError;
}

// [*] nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer::SetInitialLayerDetails()
{
	Input_xPool = Input_yPool = Output_xPool = Output_yPool = 1;
	wFullMask_Visible = wFullMask_Hidden = wFullMask_Pooling = 1;
	hFullMask_Visible = hFullMask_Hidden = hFullMask_Pooling = 1;
}

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer::SetRegularLayerDetails( const CSettings_Layer& PreviousLayer )
{
	Input_xPool = PreviousLayer.Output_xPool;
	Input_yPool = PreviousLayer.Output_yPool;
	Output_xPool = Input_xPool * xPool;
	Output_yPool = Input_yPool * yPool;
	wFullMask_Visible = PreviousLayer.wFullMask_Pooling;
	wFullMask_Hidden = wFullMask_Visible + Input_xPool * (wMask - 1);
	wFullMask_Pooling = wFullMask_Hidden + Input_xPool * (xPool - 1);
	hFullMask_Visible = PreviousLayer.hFullMask_Pooling;
	hFullMask_Hidden = hFullMask_Visible + Input_yPool * (hMask - 1);
	hFullMask_Pooling = hFullMask_Hidden + Input_yPool * (yPool - 1);
}

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer::LoadFromStream( istream& inStream )
{
	CBasicLayerInfo::LoadFromStream(inStream);
	inStream>>SpaCod_SparseLevel>>SpaCod_Lambda>>LearningRate;
}

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer::LoadFromStream( istream& inStream, int K_Visible )
{
	CBasicLayerInfo::LoadFromStream(inStream, K_Visible);
	inStream>>SpaCod_SparseLevel>>SpaCod_Lambda;
	inStream>>ConDiv_Decay_Mask>>ConDiv_Decay_Bias>>ConDiv_Momentum;
	inStream>>LearningRate;
}

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer::SaveToStream( ostream& outStream ) const
{
	CBasicLayerInfo::SaveToStream(outStream);
	outStream<<" "<<setprecision(3)<<fixed<<showpos<<SpaCod_SparseLevel;
	outStream<<" "<<setprecision(3)<<fixed<<showpos<<SpaCod_Lambda;
	outStream<<" "<<setprecision(3)<<fixed<<showpos<<ConDiv_Decay_Mask;
	outStream<<" "<<setprecision(3)<<fixed<<showpos<<ConDiv_Decay_Bias;
	outStream<<" "<<setprecision(3)<<fixed<<showpos<<ConDiv_Momentum;
	outStream<<endl;
	outStream<<" "<<setprecision(3)<<fixed<<showpos<<LearningRate;
	outStream<<endl;
}

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer::InitWeightItemSettings_for_RBM(
	CRestrictedBoltzmannMachine* pRBM )
{
	pRBM->SetAmountWeightItem_Edge(aMask * K_Visible * K_Hidden);
	pRBM->SetAmountWeightItem_Hidden(K_Hidden);
	pRBM->SetAmountWeightItem_Visible(K_Visible);
}

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer::InitUnitSettings_for_RBM(
	CRestrictedBoltzmannMachine* pRBM, CExtraLayerInfo& ExtraLayerInfo,
	int wSource, int hSource, RBM_FLOAT GaussianSigma_Visible )
{
	pRBM->ClearUnitSettings();

	ExtraLayerInfo.Clear();
	(*((CBasicLayerInfo*)(&ExtraLayerInfo))) = *((const CBasicLayerInfo*)this);

	ExtraLayerInfo.wVisible = wSource;
	ExtraLayerInfo.hVisible = hSource;
	ExtraLayerInfo.aVisible = ExtraLayerInfo.wVisible * ExtraLayerInfo.hVisible;
	ExtraLayerInfo.sVisible = ExtraLayerInfo.aVisible * K_Visible;
	ExtraLayerInfo.wHidden = wSource - wMask + 1;
	ExtraLayerInfo.hHidden = hSource - hMask + 1;
	ExtraLayerInfo.aHidden = ExtraLayerInfo.wHidden * ExtraLayerInfo.hHidden;
	ExtraLayerInfo.sHidden = ExtraLayerInfo.aHidden * K_Hidden;
	
	int ii, ik, jj, jk, tVI, tHI;

	bool fAreVisibleUnitsRealValued = (GaussianSigma_Visible > EPS);

	pRBM->ListUnit_Visible.Reserve(ExtraLayerInfo.sVisible);
	for ( ik = 0; ik < K_Visible; ++ik) for ( ii = 0; ii < ExtraLayerInfo.aVisible; ++ii )
	{
		tVI = pRBM->NewUnit_Visible(fAreVisibleUnitsRealValued, GaussianSigma_Visible, ik);
	}
	pRBM->ListUnit_Hidden.Reserve(ExtraLayerInfo.sHidden);
	for ( jk = 0; jk < K_Hidden; ++jk) for ( jj = 0; jj < ExtraLayerInfo.aHidden; ++jj )
	{
		tHI = pRBM->NewUnit_Hidden(false, 1.0f, jk);
	}

	//if (!ENABLE_CONNECTION_MANAGEMENT)
	//{
	//	int ix, iy, jx, jy, ox, oy, oo, tEWI;
	//	for ( ik = 0; ik < K_Hidden; ++ik ) for ( jk = 0; jk < K_Visible; ++jk ) for ( oo = 0; oo < ExtraLayerInfo.aMask; ++oo )
	//	{
	//		ox = oo % wMask;
	//		oy = oo / wMask;
	//		tEWI = oo + aMask * (jk + K_Visible * ik);
	//		for ( ii = 0; ii < ExtraLayerInfo.aHidden; ++ii )
	//		{
	//			ix = ii % ExtraLayerInfo.wHidden;
	//			iy = ii / ExtraLayerInfo.wHidden;
	//			jx = ix + ox;
	//			jy = iy + oy;
	//			jj = jx + jy * ExtraLayerInfo.wVisible;
	//			tVI = jj + jk * ExtraLayerInfo.aVisible;
	//			tHI = ii + ik * ExtraLayerInfo.aHidden;
	//			pRBM->NewRelation_Edge(tVI, tHI, tEWI);
	//		} // ii (jj)
	//	} // oo, jk, ik
	//}
}

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer::InitUnitSettings_for_MP_RBM(
	CRBM_with_MaxPooling* pMP_RBM, CExtraLayerInfo& ExtraLayerInfo,
	int wSource, int hSource, RBM_FLOAT GaussianSigma_Visible )
{
	InitUnitSettings_for_RBM(pMP_RBM, ExtraLayerInfo, wSource, hSource, GaussianSigma_Visible);
#if	FLAG_MAXPOOLING_DESERT_UNITS
	ExtraLayerInfo.wPool = ExtraLayerInfo.wHidden / xPool;
	ExtraLayerInfo.hPool = ExtraLayerInfo.hHidden / yPool;
	ExtraLayerInfo.aPool = ExtraLayerInfo.wPool * ExtraLayerInfo.hPool;
	ExtraLayerInfo.sPool = ExtraLayerInfo.aPool * ExtraLayerInfo.K_Hidden;
	pMP_RBM->InitAmountUnit_Pooling(ExtraLayerInfo.sPool);
	int xHead, xTail, yHead, yTail;
	xTail = ExtraLayerInfo.wHidden % xPool; xHead = xTail / 2;
	xTail = ExtraLayerInfo.wHidden - xTail + xHead;
	yTail = ExtraLayerInfo.hHidden % yPool; yHead = yTail / 2;
	yTail = ExtraLayerInfo.hHidden - yTail + yHead;
	int jj, jx, jy, qq, qx, qy, jqk, tHI, tPI;
	for ( jqk = 0; jqk < K_Hidden; ++jqk ) for ( jj = 0; jj < ExtraLayerInfo.aHidden; ++jj )
	{
		jx = jj % ExtraLayerInfo.wHidden; if (jx<xHead || jx>=xTail) continue;
		jy = jj / ExtraLayerInfo.wHidden; if (jy<yHead || jy>=yTail) continue;
		qx = (jx - xHead) / xPool;
		qy = (jy - yHead) / yPool;
		qq = qx + qy * ExtraLayerInfo.wPool;
		tHI = jj + jqk * ExtraLayerInfo.aHidden;
		tPI = qq + jqk * ExtraLayerInfo.aPool;
		pMP_RBM->NewRelation_MaxPooling(tHI, tPI);
	}
#else
	ExtraLayerInfo.wPool = (ExtraLayerInfo.wHidden + xPool - 1) / xPool;
	ExtraLayerInfo.hPool = (ExtraLayerInfo.hHidden + yPool - 1) / yPool;
	ExtraLayerInfo.aPool = ExtraLayerInfo.wPool * ExtraLayerInfo.hPool;
	ExtraLayerInfo.sPool = ExtraLayerInfo.aPool * ExtraLayerInfo.K_Hidden;
	pMP_RBM->InitAmountUnit_Pooling(ExtraLayerInfo.sPool);
	int jj, jx, jy, qq, qx, qy, jqk, tHI, tPI;
	for ( jqk = 0; jqk < K_Hidden; ++jqk ) for ( jj = 0; jj < ExtraLayerInfo.aHidden; ++jj )
	{
		jx = jj % ExtraLayerInfo.wHidden;
		jy = jj / ExtraLayerInfo.wHidden;
		qx = jx / xPool;
		qy = jy / yPool;
		qq = qx + qy * ExtraLayerInfo.wPool;
		tHI = jj + jqk * ExtraLayerInfo.aHidden;
		tPI = qq + jqk * ExtraLayerInfo.aPool;
		pMP_RBM->NewRelation_MaxPooling(tHI, tPI);
	}
#endif
}

// [*] nw_rbm::CConvolutionalDBN_for_Image::CSettings_CDBN

nw_rbm::CConvolutionalDBN_for_Image::CSettings_CDBN::CSettings_CDBN()
{
	Clear();
}

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_CDBN::Clear()
{
	Depth = 0;
	ListLayerSettings.Clear();
}

void nw_rbm::CConvolutionalDBN_for_Image::CSettings_CDBN::LoadFromFile( string strFN_CDBN_Settings )
{
	ifstream inFile(strFN_CDBN_Settings.c_str());
	inFile>>Depth;
	inFile>>K_Visible>>GaussianSigma_Visible;
	ListLayerSettings.Resize(Depth);
	int i_layer;

	CSettings_Layer PreviosLayerSettings;
	PreviosLayerSettings.K_Hidden = K_Visible;
	PreviosLayerSettings.SetInitialLayerDetails();
	for ( i_layer = 0; i_layer < Depth; ++i_layer )
	{
		CSettings_Layer& tLS = ListLayerSettings.pElement[i_layer];
		tLS.LoadFromStream(inFile, PreviosLayerSettings.K_Hidden);
		tLS.SetRegularLayerDetails(PreviosLayerSettings);
		PreviosLayerSettings = tLS;
	}
	inFile.close();
	inFile.clear();
}

//void nw_rbm::CConvolutionalDBN_for_Image::CSettings_CDBN::SaveToFile( string strFN_CDBN_Settings ) const
//{
//	ofstream outFile(strFN_CDBN_Settings.c_str());
//	outFile<<Depth<<endl;
//	int i_layer;
//	for ( i_layer = 0; i_layer < Depth; ++i_layer )
//		ListLayerSettings.pElement[i_layer].SaveToStream(outFile);
//	outFile.close();
//	outFile.clear();
//}

// [*] nw_rbm::CConvolutionalDBN_for_Image

nw_rbm::CConvolutionalDBN_for_Image::CConvolutionalDBN_for_Image()
{
	DeepClear();
}

void nw_rbm::CConvolutionalDBN_for_Image::Clear()
{
	CDeepBeliefNetwork::Clear();
	Settings.Clear();
	ListExtraLayerInfo.Clear();
}

void nw_rbm::CConvolutionalDBN_for_Image::DeepClear()
{
	CDeepBeliefNetwork::DeepClear();
	Settings.Clear();
	ListExtraLayerInfo.DeepClear();
}

nw_rbm::CConvolutionalDBN_for_Image::CExtraLayerInfo& nw_rbm::CConvolutionalDBN_for_Image::GetExtraLayerInfo( int LayerIndex )
{
	return ListExtraLayerInfo.pElement[LayerIndex];
}

nw_rbm::CConvolutionalDBN_for_Image::CSettings_Layer& nw_rbm::CConvolutionalDBN_for_Image::GetSingleLayerSettings( int LayerIndex )
{
	return Settings.ListLayerSettings.pElement[LayerIndex];
}

nw_rbm::CConvolutionalDBN_for_Image::CLayerMaskImages& nw_rbm::CConvolutionalDBN_for_Image::GetSingleLayerMaskImages_Hidden( int LayerIndex )
{
	return vLayerMaskImage_Hidden.pElement[LayerIndex];
}

nw_rbm::CConvolutionalDBN_for_Image::CLayerMaskImages& nw_rbm::CConvolutionalDBN_for_Image::GetSingleLayerMaskImages_Pooling( int LayerIndex )
{
	return vLayerMaskImage_Pooling.pElement[LayerIndex];
}

void nw_rbm::CConvolutionalDBN_for_Image::RefreshLayerConnectionManagers( int LayerIndex )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return;

	CConnectionManager_Proxy<CExtraLayerInfo>* pProxyCM_Visible = new CConnectionManager_Proxy<CExtraLayerInfo>;
	pProxyCM_Visible->pExecutor = ListExtraLayerInfo.pElement + LayerIndex;
	pProxyCM_Visible->pGetAmountConnection = &CExtraLayerInfo::GetAmountConnection_Visible;
	pProxyCM_Visible->pGetUnitConnectionList = &CExtraLayerInfo::GetUnitConnectionList_Visible;
	CConnectionManager_Proxy<CExtraLayerInfo>* pProxyCM_Hidden = new CConnectionManager_Proxy<CExtraLayerInfo>;
	pProxyCM_Hidden->pExecutor = ListExtraLayerInfo.pElement + LayerIndex;
	pProxyCM_Hidden->pGetAmountConnection = &CExtraLayerInfo::GetAmountConnection_Hidden;
	pProxyCM_Hidden->pGetUnitConnectionList = &CExtraLayerInfo::GetUnitConnectionList_Hidden;

	CRBM_with_MaxPooling& tLM = GetLayer(LayerIndex);
	tLM.SetConnectionManagers(pProxyCM_Visible, pProxyCM_Hidden);
}

void nw_rbm::CConvolutionalDBN_for_Image::Make_RBM_InferenceHelper(
	int LayerIndex,
	C_RBM_InferenceHelper*& p_RBM_InferenceHelper)
{
	if (LayerIndex<0 || LayerIndex>=Depth) return;

	CExtraLayerInfo& tLI = GetExtraLayerInfo(LayerIndex);
	tLI.Init_RBM_InferenceHelper(LayerIndex, p_RBM_InferenceHelper);

	return;
}

void nw_rbm::CConvolutionalDBN_for_Image::GetLayerCumulativePoolingSize_VisibleUnits(
	int LayerIndex, int& xPool, int& yPool ) const
{
	xPool = yPool = 1;
	int i_layer;

	if (LayerIndex < 0) LayerIndex = 0;
	if (LayerIndex > Settings.Depth) LayerIndex = Settings.Depth;

	const CSettings_Layer* qLS = Settings.ListLayerSettings.pElement;
	for ( i_layer = 0; i_layer < LayerIndex; ++i_layer )
	{
		xPool *= qLS->xPool;
		yPool *= qLS->yPool;
	}
}

void nw_rbm::CConvolutionalDBN_for_Image::GetLayerCumulativePoolingSize_PoolingUnits(
	int LayerIndex, int& xPool, int& yPool ) const
{
	xPool = yPool = 1;
	int i_layer;

	if (LayerIndex < 0) LayerIndex = 0;
	if (LayerIndex >= Settings.Depth) LayerIndex = Settings.Depth - 1;

	const CSettings_Layer* qLS = Settings.ListLayerSettings.pElement;
	for ( i_layer = 0; i_layer <= LayerIndex; ++i_layer )
	{
		xPool *= qLS->xPool;
		yPool *= qLS->yPool;
	}
}

void nw_rbm::CConvolutionalDBN_for_Image::InitBySettings()
{
	this->InitDepth(Settings.Depth);
	ListExtraLayerInfo.Resize(Depth);
	int i_layer;
	for ( i_layer = 0; i_layer < Depth; ++i_layer )
	{
		CSettings_Layer& tLS = GetSingleLayerSettings(i_layer);
		CRBM_with_MaxPooling& tLM = GetLayer(i_layer);
		CExtraLayerInfo& tLI = GetExtraLayerInfo(i_layer);
		tLS.InitWeightItemSettings_for_RBM(&tLM);
	}
	vLayerMaskImage_Hidden.Resize(Depth);
	vLayerMaskImage_Pooling.Resize(Depth);
}

void nw_rbm::CConvolutionalDBN_for_Image::InitByInputSize( int W, int H )
{
	int i_layer, j;
	int tw = W;
	int th = H;
	for ( i_layer = 0; i_layer < Depth; ++i_layer )
	{
		CSettings_Layer& tLS = GetSingleLayerSettings(i_layer);
		CRBM_with_MaxPooling& tLM = GetLayer(i_layer);
		CExtraLayerInfo& tLI = GetExtraLayerInfo(i_layer);
		
		if (i_layer==0)
			tLS.InitUnitSettings_for_MP_RBM(&tLM, tLI, tw, th, Settings.GaussianSigma_Visible);
		else
			tLS.InitUnitSettings_for_MP_RBM(&tLM, tLI, tw, th, 0.0f);

		tw = tLI.wPool;
		th = tLI.hPool;
	}

	for ( i_layer = 0; i_layer < Depth; ++i_layer )
		this->RefreshLayerConnectionManagers(i_layer);

	this->InitUnitMapping();
	for ( i_layer = 1; i_layer < Depth; ++i_layer )
	{
		CExtraLayerInfo& tLI = GetExtraLayerInfo(i_layer);
		for ( j = 0; j < tLI.sVisible; ++j )
			this->NewUnitsMapping(i_layer-1, j, j);
	}
}

//RBM_FLOAT nw_rbm::CConvolutionalDBN_for_Image::ComputeWeightMatrixValueRange( 
//	int LayerIndex )
//{
//	if (LayerIndex<0 || LayerIndex>=Depth) return NULL;
//	CRBM_with_MaxPooling& tLM = GetLayer(LayerIndex);
//	CRestrictedBoltzmannMachine::CList_WeightItem& wList = tLM.ListWeightItem_Edge;
//	if (wList.Size < 1) return 1.0f;
//	int i_wi;
//	RBM_FLOAT t;
//	RBM_FLOAT s1 = 0.0f;
//	RBM_FLOAT s2 = 0.0f;
//	RBM_FLOAT ret = 0.0f;
//	for ( i_wi = 0; i_wi < (int)wList.Size; ++i_wi )
//	{
//		t = wList.pElement[i_wi].Value;
//		s1 += t;
//		s2 += t * t;
//		if (ret < abs(t))
//			ret = abs(t);
//	}
//	s1 /= wList.Size; s2 /= wList.Size; s2 -= s1 * s1;
//	ret = abs(s1) + 2.0 * sqrt(s2);
//	return ret;
//}
//
//void nw_rbm::CConvolutionalDBN_for_Image::DrawWeightMatrix(
//	int LayerIndex, int VisibleDimensionIndex, int HiddenDimensionIndex,
//	RBM_FLOAT ZoomScale, CImg<RBM_FLOAT>& imgDst )
//{
//	if (LayerIndex<0 || LayerIndex>=Depth) return;
//	CRBM_with_MaxPooling& tLM = GetLayer(LayerIndex);
//	CSettings_Layer& tLS = GetSingleLayerSettings(LayerIndex);
//
//	if (HiddenDimensionIndex<0 || HiddenDimensionIndex>=tLS.K_Hidden) return;
//
//	int v_begin, v_end;
//	if (VisibleDimensionIndex<0 || VisibleDimensionIndex>=tLS.K_Visible)
//	{	v_begin = 0; v_end = tLS.K_Visible; }
//	else
//	{	v_begin = VisibleDimensionIndex; v_end = v_begin + 1; }
//
//	const CWeightItem* qWI;
//
//	CImg<RBM_FLOAT> imgTmp;
//	imgTmp.assign(tLS.wMask, tLS.hMask, 1, 1);
//	imgTmp.fill(0.0f);
//	int i_pixel;
//	for ( VisibleDimensionIndex = v_begin; VisibleDimensionIndex < v_end; ++VisibleDimensionIndex )
//	{
//		qWI = this->GetWeightMatrix(LayerIndex, VisibleDimensionIndex, HiddenDimensionIndex);
//		for ( i_pixel = 0; i_pixel < tLS.aMask; ++i_pixel )
//			imgTmp.data[i_pixel] += qWI[i_pixel].Value;
//	}
//	VisibleDimensionIndex = v_end - v_begin;
//	for ( i_pixel = 0; i_pixel < tLS.aMask; ++i_pixel )
//		imgTmp.data[i_pixel] /= VisibleDimensionIndex;
//	imgDst = imgTmp.resize((int)(-100*ZoomScale), (int)(-100*ZoomScale), -100, -100, 3);
//}
//
//void nw_rbm::CConvolutionalDBN_for_Image::DrawNormalizedWeightMatrix(
//	int LayerIndex, int VisibleDimensionIndex, int HiddenDimensionIndex,
//	RBM_FLOAT ZoomScale, RBM_FLOAT MaxValue, CImg<int>& imgDst )
//{
//	CImg<RBM_FLOAT> imgTmp;
//	DrawWeightMatrix(LayerIndex, VisibleDimensionIndex, HiddenDimensionIndex, ZoomScale, imgTmp);
//	CExternalTools::NormalizeToGeneralIntGrayscaleImage(imgTmp, imgDst, -MaxValue, +MaxValue, true);
//}

void nw_rbm::CConvolutionalDBN_for_Image::SetVisibleState( int DimensionIndex, const RBM_FLOAT* qSrc )
{
	CRBM_with_MaxPooling& tLM = GetLayer(0);
	CExtraLayerInfo& tLI = GetExtraLayerInfo(0);
	if (DimensionIndex<0 || DimensionIndex>=tLI.K_Visible) return;

	int i;
	CUnitBasic* pUV = tLM.ListUnit_Visible.pElement;
	for ( i = 0; i < tLI.sVisible; ++i )
		pUV[i].State = qSrc[i];
}

const nw_rbm::CUnitBasic* nw_rbm::CConvolutionalDBN_for_Image::GetUnits_Visible(
	int LayerIndex, int VisibleDimensionIndex,
	int& DstW, int& DstH )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return NULL;
	CRBM_with_MaxPooling& tLM = GetLayer(LayerIndex);
	CExtraLayerInfo& tLI = GetExtraLayerInfo(LayerIndex);
	if (VisibleDimensionIndex<0 || VisibleDimensionIndex>=tLI.K_Visible) return NULL;
	DstW = tLI.wVisible; DstH = tLI.hVisible;
	return tLM.ListUnit_Visible.pElement
		+ 0 + tLI.aVisible * VisibleDimensionIndex;
}

const nw_rbm::CUnitBasic* nw_rbm::CConvolutionalDBN_for_Image::GetUnits_Hidden(
	int LayerIndex, int HiddenDimensionIndex,
	int& DstW, int& DstH )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return NULL;
	CRBM_with_MaxPooling& tLM = GetLayer(LayerIndex);
	CExtraLayerInfo& tLI = GetExtraLayerInfo(LayerIndex);
	if (HiddenDimensionIndex<0 || HiddenDimensionIndex>=tLI.K_Hidden) return NULL;
	DstW = tLI.wHidden; DstH = tLI.hHidden;
	return tLM.ListUnit_Hidden.pElement
		+ 0 + tLI.aHidden * HiddenDimensionIndex;
}

const nw_rbm::CUnitPooling* nw_rbm::CConvolutionalDBN_for_Image::GetUnits_Pooling(
	int LayerIndex, int HiddenDimensionIndex,
	int& DstW, int& DstH )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return NULL;
	CRBM_with_MaxPooling& tLM = GetLayer(LayerIndex);
	CExtraLayerInfo& tLI = GetExtraLayerInfo(LayerIndex);
	if (HiddenDimensionIndex<0 || HiddenDimensionIndex>=tLI.K_Hidden) return NULL;
	DstW = tLI.wPool; DstH = tLI.hPool;
	return tLM.ListUnit_Pooling.pElement
		+ 0 + tLI.aPool * HiddenDimensionIndex;
}

const nw_rbm::CWeightItem* nw_rbm::CConvolutionalDBN_for_Image::GetWeightMatrix(
	int LayerIndex, int VisibleDimensionIndex, int HiddenDimensionIndex)
{
	if (LayerIndex<0 || LayerIndex>=Depth) return NULL;
	CRBM_with_MaxPooling& tLM = GetLayer(LayerIndex);
	CSettings_Layer& tLS = GetSingleLayerSettings(LayerIndex);
	if (VisibleDimensionIndex<0 || VisibleDimensionIndex>=tLS.K_Visible) return NULL;
	if (HiddenDimensionIndex<0 || HiddenDimensionIndex>=tLS.K_Hidden) return NULL;
	//DstW = tLS.wMask; DstH = tLS.hMask;
	return tLM.ListWeightItem_Edge.pElement
		+ 0 + tLS.aMask * (VisibleDimensionIndex + tLS.K_Visible * HiddenDimensionIndex);
}

void nw_rbm::CConvolutionalDBN_for_Image::ComputeMaskImages(
	int LayerIndex )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return;
	CRBM_with_MaxPooling& tLM = GetLayer(LayerIndex);
	CSettings_Layer& tLS = GetSingleLayerSettings(LayerIndex);

	CLayerMaskImages& tMI_H = vLayerMaskImage_Hidden.pElement[LayerIndex];
	tMI_H.W = tLS.wFullMask_Hidden;
	tMI_H.H = tLS.hFullMask_Hidden;
	tMI_H.A = tMI_H.W * tMI_H.H;
	tMI_H.V = Settings.K_Visible;
	tMI_H.U = tLS.K_Hidden;
	tMI_H.B = tMI_H.V * tMI_H.U;
	tMI_H.SS = tMI_H.A * tMI_H.B;
	tMI_H.vmatData.Resize(tMI_H.SS); tMI_H.vmatData.Fill(0.0f);
	tMI_H.vmatMean.Resize(tMI_H.B); tMI_H.vmatMean.Fill(0.0f);
	tMI_H.vmatVar.Resize(tMI_H.B); tMI_H.vmatVar.Fill(0.0f);
	CLayerMaskImages& tMI_P = vLayerMaskImage_Pooling.pElement[LayerIndex];
	tMI_P.W = tLS.wFullMask_Pooling;
	tMI_P.H = tLS.hFullMask_Pooling;
	tMI_P.A = tMI_P.W * tMI_P.H;
	tMI_P.V = Settings.K_Visible;
	tMI_P.U = tLS.K_Hidden;
	tMI_P.B = tMI_P.V * tMI_P.U;
	tMI_P.SS = tMI_P.A * tMI_P.B;
	tMI_P.vmatData.Resize(tMI_P.SS); tMI_P.vmatData.Fill(0.0f);
	tMI_P.vmatMean.Resize(tMI_P.B); tMI_P.vmatMean.Fill(0.0f);
	tMI_P.vmatVar.Resize(tMI_P.B); tMI_P.vmatVar.Fill(0.0f);

	int i_vv, i_uu, i_kV, i_pixel, ox, oy, px, py, pp;
	const CWeightItem* q_tWI;
	RBM_FLOAT tw;
	RBM_FLOAT* pD;
	RBM_FLOAT* pWS;
	const RBM_FLOAT* qD;
	CSimpleTypeArray<RBM_FLOAT> matWeightSum_Hidden;
	CSimpleTypeArray<RBM_FLOAT> matWeightSum_Pooling;

	matWeightSum_Hidden.Resize(tMI_H.A);
	matWeightSum_Pooling.Resize(tMI_P.A);
	for ( i_uu = 0; i_uu < tMI_H.U; ++i_uu ) for ( i_vv = 0; i_vv < tMI_H.V; ++i_vv )
	{
		pD = tMI_H.vmatData.pElement + 0 + tMI_H.A * (i_vv + tMI_H.V * i_uu);
		pWS = matWeightSum_Hidden.pElement;
		for ( i_pixel = 0; i_pixel < tMI_H.A; ++i_pixel )
		{
			pD[i_pixel] = 0.0f;
			pWS[i_pixel] = 0.0f;
		} // i_pixel
		if (LayerIndex == 0)
		{
			q_tWI = this->GetWeightMatrix(LayerIndex, i_vv, i_uu);
			for ( i_pixel = 0; i_pixel < tMI_H.A; ++i_pixel )
			{
				tw = q_tWI[i_pixel].Value;
				pD[i_pixel] += tw;
				pWS[i_pixel] += 1.0f;
			} // i_pixel
		} // (LayerIndex == 0)
		else
		{
			CLayerMaskImages& sMI_P = vLayerMaskImage_Pooling.pElement[LayerIndex-1];
			//CSettings_Layer& sLS = GetSingleLayerSettings(LayerIndex-1);
			for ( i_kV = 0; i_kV < tLS.K_Visible; ++i_kV )
			{
				q_tWI = this->GetWeightMatrix(LayerIndex, i_kV, i_uu);
				qD = sMI_P.vmatData.pElement + 0 + sMI_P.A * (i_vv + sMI_P.V * i_kV);
				for ( oy = 0; oy < tLS.hMask; ++oy ) for ( ox = 0; ox < tLS.wMask; ++ox )
				{
					tw = q_tWI[ox + tLS.wMask * oy].Value;
					if (tw < 0) tw = 0.0f;
					for ( i_pixel = 0; i_pixel < sMI_P.A; ++i_pixel )
					{
						px = i_pixel % sMI_P.W + ox * tLS.Input_xPool;
						py = i_pixel / sMI_P.W + oy * tLS.Input_yPool;
						pp = px + tMI_H.W * py;
						pD[pp] += qD[i_pixel] * tw;
						pWS[pp] += tw;
					} // i_pixel
				} // ox, oy
			} // i_kV
		} // !(LayerIndex == 0)

		if (1)
		{
			RBM_FLOAT& s1 = tMI_H.vmatMean.pElement[i_vv + tMI_H.V * i_uu];
			RBM_FLOAT& s2 = tMI_H.vmatVar.pElement[i_vv + tMI_H.V * i_uu];
			for ( i_pixel = 0; i_pixel < tMI_H.A; ++i_pixel )
			{
				tw = pWS[i_pixel];
				if (abs(tw) > EPS)
					tw = pD[i_pixel] / tw;
				else
					tw = pD[i_pixel];
				pD[i_pixel] = tw;
				pWS[i_pixel] = tw;
				s1 += tw;
				s2 += tw * tw;
			} // i_pixel
			s1 /= tMI_H.A;
			s2 /= tMI_H.A;
			s2 -= s1 * s1;
			s2 = (s2<0.0f ? 0.0f : sqrt(s2));
		} // (1)

		pD = tMI_P.vmatData.pElement + 0 + tMI_P.A * (i_vv + tMI_P.V * i_uu);
		pWS = matWeightSum_Pooling.pElement;
		for ( i_pixel = 0; i_pixel < tMI_P.A; ++i_pixel )
		{
			pD[i_pixel] = 0.0f;
			pWS[i_pixel] = 0.0f;
		} // i_pixel
		qD = tMI_H.vmatData.pElement + 0 + tMI_H.A * (i_vv + tMI_H.V * i_uu);
		for ( oy = 0; oy < tLS.yPool; ++oy ) for ( ox = 0; ox < tLS.xPool; ++ox )
		{
			for ( i_pixel = 0; i_pixel < tMI_H.A; ++i_pixel )
			{
				px = i_pixel % tMI_H.W + ox * tLS.Input_xPool;
				py = i_pixel / tMI_H.W + oy * tLS.Input_yPool;
				pp = px + tMI_P.W * py;
				pD[pp] += qD[i_pixel];
				pWS[pp] += 1.0f;
			} // i_pixel
		} // ox, oy

		if (1)
		{
			RBM_FLOAT& s1 = tMI_P.vmatMean.pElement[i_vv + tMI_P.V * i_uu];
			RBM_FLOAT& s2 = tMI_P.vmatVar.pElement[i_vv + tMI_P.V * i_uu];
			for ( i_pixel = 0; i_pixel < tMI_P.A; ++i_pixel )
			{
				tw = pWS[i_pixel];
				if (abs(tw) > EPS)
					tw = pD[i_pixel] / tw;
				else
					tw = pD[i_pixel];
				pD[i_pixel] = tw;
				pWS[i_pixel] = tw;
				s1 += tw;
				s2 += tw * tw;
			} // i_pixel
			s1 /= tMI_P.A;
			s2 /= tMI_P.A;
			s2 -= s1 * s1;
			s2 = (s2<0.0f ? 0.0f : sqrt(s2));
		} // (1)

	} // i_uu, i_vv

////	CLayerMaskImages& tMI = vLayerMaskImage_Hidden.pElement[LayerIndex];
////	if (LayerIndex == 0)
////	{
////		tMI.W = tLS.wMask;
////		tMI.H = tLS.hMask;
////		tMI.A = tMI.W * tMI.H;
////		tMI.U = tLS.K_Hidden;
////		tMI.V = tLS.K_Visible;
////		tMI.B = tMI.U * tMI.V;
////		tMI.SS = tMI.A * tMI.B;
////	}
////	else
////	{
////		CLayerMaskImages& qMI = vLayerMaskImage_Hidden.pElement[LayerIndex - 1];
////		CSettings_Layer& qLS = GetSingleLayerSettings(LayerIndex - 1);
////		tMI.W = qMI.W + tLS.wMask * qLS.xPool - 1;
////		tMI.H = qMI.H + tLS.hMask * qLS.yPool - 1;
////		tMI.A = tMI.W * tMI.H;
////		tMI.U = tLS.K_Hidden;
////		tMI.V = qMI.V;
////		tMI.B = tMI.U * tMI.V;
////		tMI.SS = tMI.A * tMI.B;
////	}
////	tMI.vmatData.Clear(); tMI.vmatData.Resize(tMI.SS); tMI.vmatData.Fill(0.0f);
////	tMI.vmatMean.Clear(); tMI.vmatMean.Resize(tMI.B); tMI.vmatMean.Fill(0.0f);
////	tMI.vmatVar.Clear(); tMI.vmatVar.Resize(tMI.B); tMI.vmatVar.Fill(0.0f);
////
////	int i_k_h, i_v, i_pixel, i_k_v, i_v_x, i_v_y, i_v_u, i_v_v;
////	int XX, YY, UU, VV, tx, ty;
////	const CWeightItem* q_tWI;
////	RBM_FLOAT tw, tdata;
////
////	CImg<RBM_FLOAT> imgTempSumWeight;
////	imgTempSumWeight.assign(tMI.W, tMI.H);
////	imgTempSumWeight.fill(0.0f);
////	RBM_FLOAT* p_matSumWeight = imgTempSumWeight.data;
////
////	for ( i_k_h = 0; i_k_h < tMI.U; ++i_k_h ) for ( i_v = 0; i_v < tMI.V; ++i_v )
////	{
////		RBM_FLOAT* pD = tMI.vmatData.pElement
////			+ 0 + tMI.A * (i_v + tMI.V * i_k_h);
////		tw = 0.0f;
////		//tw = tLM.ListWeightItem_Hidden.pElement[i_k_h].Value;
////		for ( i_pixel = 0; i_pixel < tMI.A; ++i_pixel )
////			pD[i_pixel] = tw;
////
////		//CImg<int> imgOutput;
////		CImg<RBM_FLOAT> imgTmp_f1;
////		imgTmp_f1.assign(1,1);
////		imgTmp_f1.fill(0);
////
////		if (LayerIndex == 0)
////		{
////			q_tWI = this->GetWeightMatrix(LayerIndex, i_v, i_k_h);
////
////			//imgTmp_f1.assign(tMI.W, tMI.H);
////			//imgTmp_f1.fill(0.0f);
////
////			imgTempSumWeight.fill(1.0f);
////			for ( i_pixel = 0; i_pixel < tMI.A; ++i_pixel )
////			{
////				tdata = q_tWI[i_pixel].Value;
////
////				//imgTmp_f1.data[i_pixel] = tdata;
////
////				pD[i_pixel] += tdata;
////			}
////		}
////		else
////		{
////			CLayerMaskImages& qMI = vLayerMaskImage_Hidden.pElement[LayerIndex - 1];
////			CSettings_Layer& qLS = GetSingleLayerSettings(LayerIndex - 1);
////
////			imgTempSumWeight.fill(0.0f);
////			for ( i_k_v = 0; i_k_v < qMI.U; ++i_k_v )
////			{
////				const RBM_FLOAT* qD = qMI.vmatData.pElement
////					+ 0 + qMI.A * (i_v + qMI.V * i_k_v);
////				q_tWI = this->GetWeightMatrix(LayerIndex, i_k_v, i_k_h);
////
////				//imgTmp_f1.assign(qMI.W, qMI.H);
////				//imgTmp_f1.fill(0.0f);
////
////				XX = tLS.wMask * qLS.xPool;
////				YY = tLS.hMask * qLS.yPool;
////				for ( i_v_y = 0; i_v_y < YY; ++i_v_y ) for ( i_v_x = 0; i_v_x < XX; ++i_v_x )
////				{
////					tx = i_v_x / tLS.xPool; ty = i_v_y / tLS.yPool;
////					tw = q_tWI[tx + tLS.wMask * ty].Value;
////
////					tw = (tw<0 ? 0 : tw);
////					//tw = CMyMath::my_exp(tw);
////
////					UU = qMI.W; VV = qMI.H;
////					for ( i_v_v = 0; i_v_v < VV; ++i_v_v ) for ( i_v_u = 0; i_v_u < UU; ++i_v_u )
////					{
////						tx = i_v_x + i_v_u;
////						ty = i_v_y + i_v_v;
////						tdata = tw * qD[i_v_u + UU * i_v_v];
////
////						////imgTmp_f1.at(tx, ty) += tdata;
////						//imgTmp_f1.at(i_v_u, i_v_v) = tdata / tw;
////
////						pD[tx + tMI.W * ty] += tdata;
////						p_matSumWeight[tx + tMI.W * ty] += tw;
////					} // i_v_u, i_v_v
////				} // i_v_x, i_v_y
////
////				////XX = tLS.wMask; YY = tLS.hMask;
////				////imgTmp_f1.assign(XX, YY);
////				////for ( i_v_y = 0; i_v_y < YY; ++i_v_y ) for ( i_v_x = 0; i_v_x < XX; ++i_v_x )
////				////	imgTmp_f1.at(i_v_x, i_v_y) = q_tWI[i_v_x + XX * i_v_y].Value;
////
////				//if (LayerIndex==1 && i_k_h==3)
////				//	imgTmp_f1.display();
////
////			} // i_k_h
////
////		} // (LayerIndex == 0)
////
////		//if (LayerIndex==0 && i_k_h==3)
////		//	imgTmp_f1.display();
////
////		for ( i_pixel = 0; i_pixel < tMI.A; ++i_pixel )
////		{
////			tw = p_matSumWeight[i_pixel];
////			tdata = pD[i_pixel];
////			if (tw<-MINIMUM_DOUBLE_EPS_C || tw>+MINIMUM_DOUBLE_EPS_C)
////				tdata /= tw;
////			pD[i_pixel] = p_matSumWeight[i_pixel] = tdata;
////		}
////
////		//if (LayerIndex==1 && i_k_h==15)
////		//	imgTempSumWeight.display();
////
////		i_v = i_v;
////
////		RBM_FLOAT& s1 = tMI.vmatMean.pElement[i_v + tMI.V * i_k_h];
////		RBM_FLOAT& s2 = tMI.vmatVar.pElement[i_v + tMI.V * i_k_h];
////
////		for ( i_pixel = 0; i_pixel < tMI.A; ++i_pixel )
////		{
////			tw = pD[i_pixel];
////			s1 += tw;
////			s2 += tw * tw;
////		}
////		s1 = s1;
////		s2 = s2;
////	} // i_k_h, i_v
////
////	for ( i_k_h = 0; i_k_h < tMI.U; ++i_k_h ) for ( i_v = 0; i_v < tMI.V; ++i_v )
////	{
////		RBM_FLOAT& s1 = tMI.vmatMean.pElement[i_v + tMI.V * i_k_h];
////		RBM_FLOAT& s2 = tMI.vmatVar.pElement[i_v + tMI.V * i_k_h];
////		s1 /= tMI.A;
////		s2 /= tMI.A;
////		s2 -= s1 * s1;
////		s2 = (s2<0.0f ? 0.0f : sqrt(s2));
////	}
}

void nw_rbm::CConvolutionalDBN_for_Image::DrawMaskImage(
	int LayerIndex, int ChannelIndex, int GroupIndex,
	RBM_FLOAT ZoomScale, CImg<RBM_FLOAT>& imgDst )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return;
	CLayerMaskImages& tMI = vLayerMaskImage_Hidden.pElement[LayerIndex];
	if (ChannelIndex<0 || ChannelIndex>=tMI.V) return;
	if (GroupIndex<0 || GroupIndex>=tMI.U) return;
	const RBM_FLOAT* qD = tMI.vmatData.pElement
		+ 0 + tMI.A * (ChannelIndex + tMI.V * GroupIndex);

	CImg<RBM_FLOAT> imgTmp;
	int i_pixel;
	imgTmp.assign(tMI.W, tMI.H, 1, 1);
	for ( i_pixel = 0; i_pixel < tMI.A; ++i_pixel )
		imgTmp.data[i_pixel] = qD[i_pixel];
	imgDst = imgTmp.resize((int)(-100*ZoomScale), (int)(-100*ZoomScale), -100, -100, 3);
}

void nw_rbm::CConvolutionalDBN_for_Image::DrawMaskImage(
	int LayerIndex, int ChannelIndex, int GroupIndex,
	RBM_FLOAT ZoomScale, RBM_FLOAT MinValue, RBM_FLOAT MaxValue, CImg<int>& imgDst )
{
	CImg<RBM_FLOAT> imgTmp;
	DrawMaskImage(LayerIndex, ChannelIndex, GroupIndex, ZoomScale, imgTmp);
	CExternalTools::NormalizeToGeneralIntGrayscaleImage(imgTmp, imgDst, MinValue, MaxValue, true);
}

void nw_rbm::CConvolutionalDBN_for_Image::DrawMaskImage(
	int LayerIndex, int ChannelIndex, int GroupIndex,
	RBM_FLOAT ZoomScale, CImg<int>& imgDst )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return;
	CLayerMaskImages& tMI = vLayerMaskImage_Hidden.pElement[LayerIndex];
	if (ChannelIndex<0 || ChannelIndex>=tMI.V) return;
	if (GroupIndex<0 || GroupIndex>=tMI.U) return;
	int ti = ChannelIndex + tMI.V * GroupIndex;
	RBM_FLOAT tm = tMI.vmatMean.pElement[ti];
	RBM_FLOAT tv = tMI.vmatVar.pElement[ti];
	DrawMaskImage(LayerIndex, ChannelIndex, GroupIndex,
		ZoomScale, tm - tv * 2.0f, tm + tv * 2.0f, imgDst);
}

void nw_rbm::CConvolutionalDBN_for_Image::DrawUnitsPropertyImage(
	int LayerIndex, int Mode, int DimensionIndex,
	RBM_FLOAT ZoomScale, CImg<RBM_FLOAT>& imgDst)
{
	CImg<RBM_FLOAT> imgTmp;
	int i_pixel, tw, th, xcp, ycp;

	if (0)
	{
		;
	}
	else if (Mode == DRAW_UNITS_PROPERTY_MODE_VisibleState)
	{
		const nw_rbm::CUnitBasic* qUV = this->GetUnits_Visible(LayerIndex, DimensionIndex, tw, th);
		if (qUV == NULL) return;
		this->GetLayerCumulativePoolingSize_VisibleUnits(LayerIndex, xcp, ycp);
		imgTmp.assign(tw, th);
		for ( i_pixel = tw * th - 1; i_pixel >= 0; --i_pixel )
			imgTmp.data[i_pixel] = qUV[i_pixel].State;
	}
	else if (Mode == DRAW_UNITS_PROPERTY_MODE_HiddenExpectation)
	{
		const nw_rbm::CUnitBasic* qUH = this->GetUnits_Hidden(LayerIndex, DimensionIndex, tw, th);
		if (qUH == NULL) return;
		this->GetLayerCumulativePoolingSize_VisibleUnits(LayerIndex, xcp, ycp);
		imgTmp.assign(tw, th);
		for ( i_pixel = tw * th - 1; i_pixel >= 0; --i_pixel )
			imgTmp.data[i_pixel] = qUH[i_pixel].Expectation;
	}
	else if (Mode == DRAW_UNITS_PROPERTY_MODE_PoolingExpectation)
	{
		const nw_rbm::CUnitPooling* qUP = this->GetUnits_Pooling(LayerIndex, DimensionIndex, tw, th);
		if (qUP == NULL) return;
		this->GetLayerCumulativePoolingSize_PoolingUnits(LayerIndex, xcp, ycp);
		imgTmp.assign(tw, th);
		for ( i_pixel = tw * th - 1; i_pixel >= 0; --i_pixel )
			imgTmp.data[i_pixel] = qUP[i_pixel].Expectation;
	}
	else
	{
		return;
	}

	imgDst = imgTmp.resize(
		(int)(-100*ZoomScale*xcp), (int)(-100*ZoomScale*ycp), -100, -100, 5);
}

void nw_rbm::CConvolutionalDBN_for_Image::DrawUnitsPropertyImage(
	int LayerIndex, int Mode, int DimensionIndex,
	RBM_FLOAT ZoomScale, CImg<int>& imgDst)
{
	if (LayerIndex<0 || LayerIndex>=Depth) return;
	CExtraLayerInfo& tzLI = GetExtraLayerInfo(0);
	int sw = (int)(tzLI.wVisible * ZoomScale);
	int sh = (int)(tzLI.hVisible * ZoomScale);
	imgDst.assign(sw, sh, 1, 1);
	imgDst.fill(128);

	CImg<RBM_FLOAT> imgTmp;
	DrawUnitsPropertyImage(LayerIndex, Mode, DimensionIndex, ZoomScale, imgTmp);
	int tw = imgTmp.width;
	int th = imgTmp.height;

	int i_pixel, ttx, tty, ttc;	
	for ( i_pixel = tw * th - 1; i_pixel >= 0; --i_pixel )
	{
		ttx = (sw - tw) / 2 + i_pixel % tw;
		tty = (sh - th) / 2 + i_pixel / tw;
		ttc = (int)(imgTmp.data[i_pixel] * 255.0);
		imgDst.data[ttx + tty * sw] = (ttc<0 ? 0 : (ttc>255 ? 255 : ttc));
	}
}

void nw_rbm::CConvolutionalDBN_for_Image::DrawAllUnitsPropertyImage(
	int LayerIndex, int Mode,
	RBM_FLOAT ZoomScale, int xDraw, CImg<int>& imgDst)
{
	if (LayerIndex<0 || LayerIndex>=Depth) return;
	CExtraLayerInfo& tzLI = GetExtraLayerInfo(0);
	int sw = (int)(tzLI.wVisible * ZoomScale);
	int sh = (int)(tzLI.hVisible * ZoomScale);

	CExtraLayerInfo& tLI = GetExtraLayerInfo(LayerIndex);

	int tK = 0;
	if (0)
	{
		;
	}
	else if (Mode == DRAW_UNITS_PROPERTY_MODE_VisibleState)
		tK = tLI.K_Visible;
	else if (Mode == DRAW_UNITS_PROPERTY_MODE_HiddenExpectation)
		tK = tLI.K_Hidden;
	else if (Mode == DRAW_UNITS_PROPERTY_MODE_PoolingExpectation)
		tK = tLI.K_Hidden;
	else
	{
		return;
	}


	if (xDraw < 1)
	{
		xDraw = (int)(ceil(sqrt(sw * sh * (tK + tzLI.K_Visible) * 1.0f) / sw)+0.1);
	}
	int yDraw = (tK + tzLI.K_Visible + xDraw - 1) / xDraw;
	int i_p_d;
	CImg<int> imgTmp;

	imgDst.assign(sw * xDraw, sh * yDraw, 1, 1);
	imgDst.fill(128);
	for ( i_p_d = 0; i_p_d < tzLI.K_Visible; ++i_p_d )
	{
		this->DrawUnitsPropertyImage(0, DRAW_UNITS_PROPERTY_MODE_VisibleState, i_p_d, ZoomScale, imgTmp);
		imgDst.draw_image(imgTmp,
			(tK + i_p_d) % xDraw * sw,
			(tK + i_p_d) / xDraw * sh);
	}
	for ( i_p_d = 0; i_p_d < tK; ++i_p_d )
	{
		this->DrawUnitsPropertyImage(LayerIndex, Mode, i_p_d, ZoomScale, imgTmp);
		imgDst.draw_image(imgTmp,
			i_p_d % xDraw * sw,
			i_p_d / xDraw * sh);
	}
}

void nw_rbm::CConvolutionalDBN_for_Image::ComputeBothGradient(
	int LayerIndex,
	RBM_FLOAT SpaCod_SparseLevel,
	RBM_FLOAT ExampleWeight, RBM_FLOAT GradientAmendment )
{
	CDeepBeliefNetwork::ComputeBothGradient(
		LayerIndex, SpaCod_SparseLevel, ExampleWeight, GradientAmendment);
}

void nw_rbm::CConvolutionalDBN_for_Image::UpdateBothGradient(
	int LayerIndex,
	RBM_FLOAT LearningRate, RBM_FLOAT SpaCod_Lambda,
	RBM_FLOAT ConDiv_Decay_Mask, RBM_FLOAT ConDiv_Decay_Bias, RBM_FLOAT ConDiv_Momentum)
{
	CDeepBeliefNetwork::UpdateBothGradient(
		LayerIndex,
		LearningRate, SpaCod_Lambda,
		ConDiv_Decay_Mask, ConDiv_Decay_Bias, ConDiv_Momentum);
}

RBM_FLOAT nw_rbm::CConvolutionalDBN_for_Image::GetCumulativeCosts(
	int LayerIndex,
	RBM_FLOAT SpaCod_Lambda,
	RBM_FLOAT& VisibleError, RBM_FLOAT& HiddenPenalty )
{
	return CDeepBeliefNetwork::GetCumulativeCosts(LayerIndex, SpaCod_Lambda, VisibleError, HiddenPenalty);
}

void nw_rbm::CConvolutionalDBN_for_Image::ComputeBothGradient(
	int LayerIndex,
	RBM_FLOAT ExampleWeight )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return;
	CSettings_Layer& tLS = GetSingleLayerSettings(LayerIndex);
	CExtraLayerInfo& tLI = GetExtraLayerInfo(LayerIndex);

	this->ComputeBothGradient(
		LayerIndex,
		tLS.SpaCod_SparseLevel,
		ExampleWeight,
#if FLAG_AVERAGE_WEIGHT_GRADIENT
		(RBM_FLOAT)1.0f);
#else
		(RBM_FLOAT)(1.0f / tLI.aHidden));
#endif
}

void nw_rbm::CConvolutionalDBN_for_Image::UpdateBothGradient(
	int LayerIndex )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return;
	CSettings_Layer& tLS = GetSingleLayerSettings(LayerIndex);
	this->UpdateBothGradient(
		LayerIndex,
		tLS.LearningRate,
		tLS.SpaCod_Lambda,
		tLS.ConDiv_Decay_Mask, tLS.ConDiv_Decay_Bias, tLS.ConDiv_Momentum);
}

RBM_FLOAT nw_rbm::CConvolutionalDBN_for_Image::GetCumulativeCosts(
	int LayerIndex,
	RBM_FLOAT& VisibleError, RBM_FLOAT& HiddenPenalty )
{
	if (LayerIndex<0 || LayerIndex>=Depth) return 0.0f;
	CSettings_Layer& tLS = GetSingleLayerSettings(LayerIndex);
	return this->GetCumulativeCosts(LayerIndex, tLS.SpaCod_Lambda, VisibleError, HiddenPenalty);
}

